{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# DEBUG_ONLY_TRAIN_SWITCHER_ON = False\n",
    "\n",
    "# TRAIN_CLASSIFY_MODEL = 'VGG' # VGG, DenseNet, Inception, ResNet\n",
    "# TRAIN_CLASSIFY_LEARNING_RATE = 1e-4\n",
    "# USE_SIMPLIFIED_VGG = True\n",
    "# TRAIN_CLASSIFY_USE_BN = False\n",
    "# CLASSIFY_INPUT_WIDTH, CLASSIFY_INPUT_HEIGHT, CLASSIFY_INPUT_DEPTH, CLASSIFY_INPUT_CHANNEL = 32, 32, 32, 1\n",
    "# CLASSIFY_LOG_DIR = LOG_BASE_PATH + '/classify-run-{}'\n",
    "# CLASSIFY_POSITIVE_SAMPLE_RATIO = 0.5\n",
    "# TRAIN_CLASSIFY_SAMPLE_RANDOM_OFFSET = 2\n",
    "# TRAIN_CLASSIFY_EPOCHS = 100000000\n",
    "# TRAIN_CLASSIFY_EARLY_STOPPING_PATIENCE = 10\n",
    "# TRAIN_CLASSIFY_TRAIN_BATCH_SIZE = 16\n",
    "# TRAIN_CLASSIFY_STEPS_PER_EPOCH = 10\n",
    "# TRAIN_CLASSIFY_VALID_BATCH_SIZE = 16\n",
    "# TRAIN_CLASSIFY_VALID_STEPS = 2\n",
    "\n",
    "#  9/10 [==========================>...] - ETA: 11s - loss: 0.2611 - \n",
    "# acc: 0.8889Epoch 00023: val_loss improved from 0.13012 to 0.08889, \n",
    "#         saving model to /output/training_logs/classify-run-23-14/checkpoint-23-0.0889.hdf5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "get meta_dict from cache\n",
      "tumor record size (975, 12)\n",
      "start classify_train\n",
      "classify train round 23-14\n",
      "Epoch 1/100000000\n",
      " 9/10 [==========================>...] - ETA: 19s - loss: 0.7486 - acc: 0.4861Epoch 00000: val_loss improved from inf to 0.60340, saving model to /output/training_logs/classify-run-23-14/checkpoint-00-0.6034.hdf5\n",
      "10/10 [==============================] - 310s - loss: 0.7481 - acc: 0.4875 - val_loss: 0.6034 - val_acc: 0.8438\n",
      "Epoch 2/100000000\n",
      " 3/10 [========>.....................] - ETA: 6s - loss: 0.7104 - acc: 0.6250 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (2.673968). Check your callbacks.\n",
      "  % delta_t_median)\n",
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.337671). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 4/10 [===========>..................] - ETA: 22s - loss: 0.7143 - acc: 0.5938"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.731042). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 13s - loss: 0.6979 - acc: 0.6319Epoch 00001: val_loss did not improve\n",
      "10/10 [==============================] - 247s - loss: 0.6931 - acc: 0.6562 - val_loss: 0.6644 - val_acc: 0.8438\n",
      "Epoch 3/100000000\n",
      " 3/10 [========>.....................] - ETA: 5s - loss: 0.6427 - acc: 0.8333"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (2.286947). Check your callbacks.\n",
      "  % delta_t_median)\n",
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.144081). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 4/10 [===========>..................] - ETA: 40s - loss: 0.6587 - acc: 0.8125"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.124150). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 14s - loss: 0.6320 - acc: 0.8333Epoch 00002: val_loss did not improve\n",
      "10/10 [==============================] - 276s - loss: 0.6367 - acc: 0.8063 - val_loss: 0.6515 - val_acc: 0.7188\n",
      "Epoch 4/100000000\n",
      " 9/10 [==========================>...] - ETA: 11s - loss: 0.6526 - acc: 0.7569Epoch 00003: val_loss did not improve\n",
      "10/10 [==============================] - 231s - loss: 0.6480 - acc: 0.7562 - val_loss: 0.6405 - val_acc: 0.6875\n",
      "Epoch 5/100000000\n",
      " 3/10 [========>.....................] - ETA: 5s - loss: 0.6303 - acc: 0.8333"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.825045). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 15s - loss: 0.6124 - acc: 0.8264Epoch 00004: val_loss did not improve\n",
      "10/10 [==============================] - 275s - loss: 0.6176 - acc: 0.8125 - val_loss: 0.6392 - val_acc: 0.7500\n",
      "Epoch 6/100000000\n",
      " 3/10 [========>.....................] - ETA: 4s - loss: 0.7103 - acc: 0.6458"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.827429). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.6421 - acc: 0.7708Epoch 00005: val_loss did not improve\n",
      "10/10 [==============================] - 240s - loss: 0.6433 - acc: 0.7688 - val_loss: 0.6389 - val_acc: 0.7500\n",
      "Epoch 7/100000000\n",
      " 9/10 [==========================>...] - ETA: 11s - loss: 0.6354 - acc: 0.7986Epoch 00006: val_loss improved from 0.60340 to 0.58707, saving model to /output/training_logs/classify-run-23-14/checkpoint-06-0.5871.hdf5\n",
      "10/10 [==============================] - 229s - loss: 0.6254 - acc: 0.8125 - val_loss: 0.5871 - val_acc: 0.8750\n",
      "Epoch 8/100000000\n",
      " 9/10 [==========================>...] - ETA: 11s - loss: 0.5947 - acc: 0.8264 ETA: 38s - loss: 0.6003 - accEpoch 00007: val_loss improved from 0.58707 to 0.53241, saving model to /output/training_logs/classify-run-23-14/checkpoint-07-0.5324.hdf5\n",
      "10/10 [==============================] - 236s - loss: 0.5912 - acc: 0.8250 - val_loss: 0.5324 - val_acc: 0.8750\n",
      "Epoch 9/100000000\n",
      " 3/10 [========>.....................] - ETA: 3s - loss: 0.5644 - acc: 0.8333"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.553284). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 13s - loss: 0.5568 - acc: 0.8125Epoch 00008: val_loss improved from 0.53241 to 0.51704, saving model to /output/training_logs/classify-run-23-14/checkpoint-08-0.5170.hdf5\n",
      "10/10 [==============================] - 255s - loss: 0.5528 - acc: 0.8125 - val_loss: 0.5170 - val_acc: 0.7812\n",
      "Epoch 10/100000000\n",
      " 3/10 [========>.....................] - ETA: 5s - loss: 0.4731 - acc: 0.9167"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (2.198280). Check your callbacks.\n",
      "  % delta_t_median)\n",
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.099875). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 13s - loss: 0.5160 - acc: 0.8542Epoch 00009: val_loss improved from 0.51704 to 0.35721, saving model to /output/training_logs/classify-run-23-14/checkpoint-09-0.3572.hdf5\n",
      "10/10 [==============================] - 244s - loss: 0.5114 - acc: 0.8562 - val_loss: 0.3572 - val_acc: 0.9375\n",
      "Epoch 11/100000000\n",
      " 9/10 [==========================>...] - ETA: 13s - loss: 0.5522 - acc: 0.8264Epoch 00010: val_loss did not improve\n",
      "10/10 [==============================] - 255s - loss: 0.5343 - acc: 0.8250 - val_loss: 0.4172 - val_acc: 0.8438\n",
      "Epoch 12/100000000\n",
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.3708 - acc: 0.9236Epoch 00011: val_loss improved from 0.35721 to 0.23566, saving model to /output/training_logs/classify-run-23-14/checkpoint-11-0.2357.hdf5\n",
      "10/10 [==============================] - 237s - loss: 0.3738 - acc: 0.9062 - val_loss: 0.2357 - val_acc: 0.9375\n",
      "Epoch 13/100000000\n",
      " 3/10 [========>.....................] - ETA: 7s - loss: 0.4006 - acc: 0.8125"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.709022). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 14s - loss: 0.4091 - acc: 0.8333Epoch 00012: val_loss did not improve\n",
      "10/10 [==============================] - 260s - loss: 0.4279 - acc: 0.8250 - val_loss: 0.2981 - val_acc: 0.8750\n",
      "Epoch 14/100000000\n",
      " 1/10 [==>...........................] - ETA: 0s - loss: 0.3453 - acc: 0.7500"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.582219). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 2/10 [=====>........................] - ETA: 3s - loss: 0.3228 - acc: 0.8125"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.653939). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 3/10 [========>.....................] - ETA: 3s - loss: 0.2921 - acc: 0.8542"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.725659). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 6/10 [=================>............] - ETA: 36s - loss: 0.3098 - acc: 0.8750"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.291849). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8/10 [=======================>......] - ETA: 23s - loss: 0.3079 - acc: 0.8906"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.291854). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.3208 - acc: 0.8819Epoch 00013: val_loss did not improve\n",
      "10/10 [==============================] - 235s - loss: 0.3310 - acc: 0.8625 - val_loss: 0.2561 - val_acc: 0.9375\n",
      "Epoch 15/100000000\n",
      " 9/10 [==========================>...] - ETA: 11s - loss: 0.3358 - acc: 0.8681Epoch 00014: val_loss did not improve\n",
      "10/10 [==============================] - 248s - loss: 0.3346 - acc: 0.8750 - val_loss: 0.3779 - val_acc: 0.8438\n",
      "Epoch 16/100000000\n",
      " 3/10 [========>.....................] - ETA: 4s - loss: 0.4472 - acc: 0.7292"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.782934). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.3198 - acc: 0.8333Epoch 00015: val_loss improved from 0.23566 to 0.21473, saving model to /output/training_logs/classify-run-23-14/checkpoint-15-0.2147.hdf5\n",
      "10/10 [==============================] - 239s - loss: 0.3165 - acc: 0.8375 - val_loss: 0.2147 - val_acc: 0.9375\n",
      "Epoch 17/100000000\n",
      " 3/10 [========>.....................] - ETA: 5s - loss: 0.2103 - acc: 0.9167"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.975317). Check your callbacks.\n",
      "  % delta_t_median)\n",
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.988375). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 4/10 [===========>..................] - ETA: 26s - loss: 0.2696 - acc: 0.8906"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (2.124733). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 6/10 [=================>............] - ETA: 33s - loss: 0.2566 - acc: 0.9062"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.988378). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8/10 [=======================>......] - ETA: 22s - loss: 0.2664 - acc: 0.8906"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.179147). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 9/10 [==========================>...] - ETA: 11s - loss: 0.2723 - acc: 0.8819"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.356855). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00016: val_loss did not improve\n",
      "10/10 [==============================] - 239s - loss: 0.2651 - acc: 0.8875 - val_loss: 0.7770 - val_acc: 0.9062\n",
      "Epoch 18/100000000\n",
      " 4/10 [===========>..................] - ETA: 29s - loss: 0.1648 - acc: 0.9531"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.275305). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 14s - loss: 0.3005 - acc: 0.9236Epoch 00017: val_loss improved from 0.21473 to 0.15762, saving model to /output/training_logs/classify-run-23-14/checkpoint-17-0.1576.hdf5\n",
      "10/10 [==============================] - 262s - loss: 0.3463 - acc: 0.9062 - val_loss: 0.1576 - val_acc: 0.9688\n",
      "Epoch 19/100000000\n",
      " 3/10 [========>.....................] - ETA: 11s - loss: 0.1524 - acc: 0.9375"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (4.758627). Check your callbacks.\n",
      "  % delta_t_median)\n",
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (2.380003). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 13s - loss: 0.2886 - acc: 0.8889Epoch 00018: val_loss did not improve\n",
      "10/10 [==============================] - 243s - loss: 0.3035 - acc: 0.8750 - val_loss: 0.2965 - val_acc: 0.9062\n",
      "Epoch 20/100000000\n",
      " 3/10 [========>.....................] - ETA: 4s - loss: 0.3313 - acc: 0.8750"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.955317). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 13s - loss: 0.3094 - acc: 0.8958Epoch 00019: val_loss did not improve\n",
      "10/10 [==============================] - 259s - loss: 0.2938 - acc: 0.9062 - val_loss: 0.1751 - val_acc: 0.9688\n",
      "Epoch 21/100000000\n",
      " 2/10 [=====>........................] - ETA: 7s - loss: 0.2651 - acc: 0.9062"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.608261). Check your callbacks.\n",
      "  % delta_t_median)\n",
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.804862). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 3/10 [========>.....................] - ETA: 4s - loss: 0.2454 - acc: 0.9167"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.938801). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 4/10 [===========>..................] - ETA: 29s - loss: 0.2111 - acc: 0.9375"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.724037). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 7/10 [====================>.........] - ETA: 30s - loss: 0.2364 - acc: 0.9196"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.509273). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 11s - loss: 0.2180 - acc: 0.9306Epoch 00020: val_loss did not improve\n",
      "10/10 [==============================] - 226s - loss: 0.2339 - acc: 0.9250 - val_loss: 0.2212 - val_acc: 0.9062\n",
      "Epoch 22/100000000\n",
      " 4/10 [===========>..................] - ETA: 40s - loss: 0.4965 - acc: 0.8125"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.518959). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 5/10 [==============>...............] - ETA: 48s - loss: 0.4665 - acc: 0.8250"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.036289). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 6/10 [=================>............] - ETA: 45s - loss: 0.4074 - acc: 0.8542"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.561746). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 14s - loss: 0.3288 - acc: 0.8819Epoch 00021: val_loss improved from 0.15762 to 0.13012, saving model to /output/training_logs/classify-run-23-14/checkpoint-21-0.1301.hdf5\n",
      "10/10 [==============================] - 267s - loss: 0.3310 - acc: 0.8812 - val_loss: 0.1301 - val_acc: 0.9688\n",
      "Epoch 23/100000000\n",
      " 2/10 [=====>........................] - ETA: 4s - loss: 0.3686 - acc: 0.8125"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.034994). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 3/10 [========>.....................] - ETA: 7s - loss: 0.3935 - acc: 0.8125"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.449565). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r",
      " 4/10 [===========>..................] - ETA: 27s - loss: 0.3338 - acc: 0.8594"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.980586). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 11s - loss: 0.3115 - acc: 0.8750"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.926178). Check your callbacks.\n",
      "  % delta_t_median)\n",
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.585073). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00022: val_loss did not improve\n",
      "10/10 [==============================] - 229s - loss: 0.3121 - acc: 0.8750 - val_loss: 0.2287 - val_acc: 0.9375\n",
      "Epoch 24/100000000\n",
      " 3/10 [========>.....................] - ETA: 3s - loss: 0.3079 - acc: 0.8958"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.604447). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 11s - loss: 0.2611 - acc: 0.8889Epoch 00023: val_loss improved from 0.13012 to 0.08889, saving model to /output/training_logs/classify-run-23-14/checkpoint-23-0.0889.hdf5\n",
      "10/10 [==============================] - 224s - loss: 0.2967 - acc: 0.8688 - val_loss: 0.0889 - val_acc: 0.9688\n",
      "Epoch 25/100000000\n",
      " 3/10 [========>.....................] - ETA: 6s - loss: 0.6502 - acc: 0.7500"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.260761). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.4046 - acc: 0.8750Epoch 00024: val_loss did not improve\n",
      "10/10 [==============================] - 222s - loss: 0.4001 - acc: 0.8750 - val_loss: 0.3216 - val_acc: 0.9062\n",
      "Epoch 26/100000000\n",
      " 3/10 [========>.....................] - ETA: 1s - loss: 0.2752 - acc: 0.9167"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.256727). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.2878 - acc: 0.9097Epoch 00025: val_loss did not improve\n",
      "10/10 [==============================] - 236s - loss: 0.2833 - acc: 0.9125 - val_loss: 0.3250 - val_acc: 0.8750\n",
      "Epoch 27/100000000\n",
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.2574 - acc: 0.9097Epoch 00026: val_loss did not improve\n",
      "10/10 [==============================] - 238s - loss: 0.2796 - acc: 0.8938 - val_loss: 0.2067 - val_acc: 0.9375\n",
      "Epoch 28/100000000\n",
      " 9/10 [==========================>...] - ETA: 10s - loss: 0.1905 - acc: 0.9444Epoch 00027: val_loss did not improve\n",
      "10/10 [==============================] - 216s - loss: 0.2051 - acc: 0.9437 - val_loss: 0.2094 - val_acc: 0.9375\n",
      "Epoch 29/100000000\n",
      " 9/10 [==========================>...] - ETA: 13s - loss: 0.3148 - acc: 0.8958 ETA: 45s - loss: 0.3083 - acc: Epoch 00028: val_loss did not improve\n",
      "10/10 [==============================] - 244s - loss: 0.3134 - acc: 0.9000 - val_loss: 0.5762 - val_acc: 0.8750\n",
      "Epoch 30/100000000\n",
      " 2/10 [=====>........................] - ETA: 5s - loss: 0.2640 - acc: 0.8750"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.128042). Check your callbacks.\n",
      "  % delta_t_median)\n",
      "/usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.564729). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.3670 - acc: 0.8611Epoch 00029: val_loss did not improve\n",
      "10/10 [==============================] - 234s - loss: 0.3628 - acc: 0.8562 - val_loss: 0.3974 - val_acc: 0.8125\n",
      "Epoch 31/100000000\n",
      " 4/10 [===========>..................] - ETA: 28s - loss: 0.3134 - acc: 0.8750"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-99cf1b398bcd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mclassify_train\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mclassify_train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mclassify_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/root/Tianchi-Medical-LungTumorDetect/classify_train.py\u001b[0m in \u001b[0;36mclassify_train\u001b[0;34m()\u001b[0m\n\u001b[1;32m     39\u001b[0m                         \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mget_classify_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTRAIN_CLASSIFY_VALID_BATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfrom_train\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTRAIN_CLASSIFY_VALID_STEPS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m                         \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTRAIN_CLASSIFY_EPOCHS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m                         callbacks=[tensorboard, checkpoint, early_stopping])\n\u001b[0m\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     87\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 88\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_support_signature\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetargspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_q_size, workers, pickle_safe, initial_epoch)\u001b[0m\n\u001b[1;32m   1857\u001b[0m                             \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1858\u001b[0m                         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1859\u001b[0;31m                             \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1860\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1861\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerator_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'__len__'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from classify_train import classify_train\n",
    "classify_train()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
